<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Jacobi: Jacobi.h File Reference</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">Jacobi
   
   </div>
   
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
      <li><a href="globals.html"><span>File&#160;Members</span></a></li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="summary">
<a href="#define-members">Defines</a> &#124;
<a href="#func-members">Functions</a>  </div>
  <div class="headertitle">
<div class="title">Jacobi.h File Reference</div>  </div>
</div><!--header-->
<div class="contents">

<p>The header containing the most relevant functions for the Jacobi solver.  
<a href="#details">More...</a></p>
<div class="textblock"><code>#include &lt;mpi.h&gt;</code><br/>
<code>#include &lt;stdio.h&gt;</code><br/>
<code>#include &lt;stdlib.h&gt;</code><br/>
<code>#include &lt;cuda_runtime.h&gt;</code><br/>
</div><div class="textblock"><div class="dynheader">
Include dependency graph for Jacobi.h:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h__incl.png" border="0" usemap="#Jacobi_8h" alt=""/></div>
<map name="Jacobi_8h" id="Jacobi_8h">
</map>
</div>
</div><div class="textblock"><div class="dynheader">
This graph shows which files directly or indirectly include this file:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h__dep__incl.png" border="0" usemap="#Jacobi_8hdep" alt=""/></div>
<map name="Jacobi_8hdep" id="Jacobi_8hdep">
<area shape="rect" id="node3" href="CUDA__Aware__MPI_8c.html" title="The implementation details for the CUDA&#45;aware MPI version." alt="" coords="5,83,152,112"/><area shape="rect" id="node5" href="CUDA__Normal__MPI_8c.html" title="The implementation details for the normal CUDA &amp; MPI version." alt="" coords="176,83,328,112"/><area shape="rect" id="node7" href="Device_8cu.html" title="This contains the device kernels as well as the host wrappers for these kernels." alt="" coords="352,83,432,112"/><area shape="rect" id="node9" href="Host_8c.html" title="This contains the host functions for data allocations, message passing and host&#45;side computations..." alt="" coords="456,83,517,112"/><area shape="rect" id="node11" href="Input_8c.html" title="This contains the command&#45;line argument parser and support functions." alt="" coords="541,83,603,112"/><area shape="rect" id="node13" href="Jacobi_8c.html" title="This contains the application entry point." alt="" coords="627,83,696,112"/></map>
</div>
</div><table class="memberdecls">
<tr><td colspan="2"><h2><a name="define-members"></a>
Defines</h2></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a5a03415e5a2d7bdc6ce680c47ddc156a">USE_FLOAT</a>&#160;&#160;&#160;0</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#ae7bacf72b151d28a9a4c55d10db0fa0b">DEFAULT_DOMAIN_SIZE</a>&#160;&#160;&#160;4096</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a05512809cad5b2530798ada1a5f06a83">MIN_DOM_SIZE</a>&#160;&#160;&#160;3</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a1cf673c1febd6ac97dd8aefadf00b5f7">ENV_LOCAL_RANK</a>&#160;&#160;&#160;&quot;MV2_COMM_WORLD_LOCAL_RANK&quot;</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a30c87e11fd7fecbcda306bb0b73ef320">MPI_MASTER_RANK</a>&#160;&#160;&#160;0</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a3bfda2a5c101b1a0762152713cda5aa8">JACOBI_TOLERANCE</a>&#160;&#160;&#160;1.0E-5F</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a3941d7542ce359c2489df632b4893aee">JACOBI_MAX_LOOPS</a>&#160;&#160;&#160;1000</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a2403320c41f08e7567cb169de5db66b3">STATUS_OK</a>&#160;&#160;&#160;0</td></tr>
<tr><td class="memItemLeft" align="right" valign="top">#define&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a23205a6a97096d57a247794f5e250963">STATUS_ERR</a>&#160;&#160;&#160;-1</td></tr>
<tr><td colspan="2"><h2><a name="func-members"></a>
Functions</h2></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a0ea9f99f85b380e912940f351366a862">Initialize</a> (int *argc, char ***argv, int *rank, int *size)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Initialize the MPI environment, allowing the CUDA device to be selected before (if necessary)  <a href="#a0ea9f99f85b380e912940f351366a862"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#ad25cf65f450abbc68de1a6409b0c92ba">Finalize</a> (real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Close (finalize) the MPI environment and deallocate buffers.  <a href="#ad25cf65f450abbc68de1a6409b0c92ba"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a243a9fb9128b70313b30d3f082cc0a61">ParseCommandLineArguments</a> (int argc, char **argv, int rank, int size, int2 *domSize, int2 *topSize, int *useFastSwap)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Parses the application's command-line arguments.  <a href="#a243a9fb9128b70313b30d3f082cc0a61"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#aef3118f365ec96030a022fa17b73fc42">ApplyTopology</a> (int *rank, int size, const int2 *topSize, int *neighbors, int2 *topIndex, MPI_Comm *cartComm)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Generates the 2D topology and establishes the neighbor relationships between MPI processes.  <a href="#aef3118f365ec96030a022fa17b73fc42"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a63442a950638ea4accb5dcb5ad7a8b87">InitializeDataChunk</a> (int topSizeY, int topIdxY, const int2 *domSize, const int *neighbors, cudaStream_t *copyStream, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real **devResidue)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This allocates and initializes all the relevant data buffers before the Jacobi run.  <a href="#a63442a950638ea4accb5dcb5ad7a8b87"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a053b144f9d3b11ad9b3563447876629f">PreRunJacobi</a> (MPI_Comm cartComm, int rank, int size, double *timerStart)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This function is called immediately before the main Jacobi loop.  <a href="#a053b144f9d3b11ad9b3563447876629f"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a52ea6f92f186de790a55fdf700026dd9">RunJacobi</a> (MPI_Comm cartComm, int rank, int size, const int2 *domSize, const int2 *topIndex, const int *neighbors, int useFastSwap, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream, int *iterations, double *avgTransferTime)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This is the main Jacobi loop, which handles device computation and data exchange between MPI processes.  <a href="#a52ea6f92f186de790a55fdf700026dd9"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a29316e4a0e49080875975ead1809fdbb">PostRunJacobi</a> (MPI_Comm cartComm, int rank, int size, const int2 *topSize, const int2 *domSize, int iterations, int useFastSwap, double timerStart, double avgTransferTime)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This function is called immediately after the main Jacobi loop.  <a href="#a29316e4a0e49080875975ead1809fdbb"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top"><a class="anchor" id="ac7925a5469b04c01679b20577671c975"></a><!-- doxytag: member="Jacobi.h::SetDeviceBeforeInit" ref="ac7925a5469b04c01679b20577671c975" args="()" -->
void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#ac7925a5469b04c01679b20577671c975">SetDeviceBeforeInit</a> ()</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This allows the MPI process to set the CUDA device before the MPI environment is initialized For the CUDA-aware MPI version, the is the only place where the device gets set. In order to do this, we rely on the node's local rank, as the MPI environment has not been initialized yet. <br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#aa8cb031e8cba7840a96582f530283fd5">SetDeviceAfterInit</a> (int rank)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This allows the MPI process to set the CUDA device after the MPI environment is initialized For the CUDA-aware MPI version, there is nothing to be done here.  <a href="#aa8cb031e8cba7840a96582f530283fd5"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a1dc0674ebcbca5a781885f2204d78251">ExchangeHalos</a> (MPI_Comm cartComm, real *devSend, real *hostSend, real *hostRecv, real *devRecv, int neighbor, int elemCount)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Exchange halo values between 2 direct neighbors This is the main difference between the normal CUDA &amp; MPI version and the CUDA-aware MPI version. In the former, the exchange first requires a copy from device to host memory, an MPI call using the host buffer and lastly, a copy of the received host buffer back to the device memory. In the latter, the host buffers are completely skipped, as the MPI environment uses the device buffers directly.  <a href="#a1dc0674ebcbca5a781885f2204d78251"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a87c5ce86d74764a06a13b73dff013870">CheckCudaCall</a> (cudaError_t command, const char *commandName, const char *fileName, int line)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">The host function for checking the result of a CUDA API call.  <a href="#a87c5ce86d74764a06a13b73dff013870"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">real&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a08324d1c9b1ee7193242670ba4b97d88">CallJacobiKernel</a> (real *devBlocks[2], real *devResidue, const int4 *bounds, const int2 *size)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">The host wrapper for one Jacobi iteration.  <a href="#a08324d1c9b1ee7193242670ba4b97d88"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#aeaebe085afbffaa0d6bcd77fa16ceb97">CopyDeviceBlock</a> (real *devBlocks[2], const int4 *bounds, const int2 *size)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">The host wrapper for copying the updated block over the old one, after a Jacobi iteration finishes.  <a href="#aeaebe085afbffaa0d6bcd77fa16ceb97"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a3b641f8b5d521a230416b64d8b809324">CopyDevHalosToBlock</a> (real *devBlock, const real *devHaloLineLeft, const real *devHaloLineRight, const int2 *size, const int *neighbors)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">The host wrapper for copying (unpacking) the values from the halo buffers to the left and right side of the data block.  <a href="#a3b641f8b5d521a230416b64d8b809324"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Jacobi_8h.html#a95a52e3c2e58d2d94403a0dce2e29686">CopyDevSideEdgesFromBlock</a> (const real *devBlock, real *devSideEdges[2], const int2 *size, const int *neighbors, cudaStream_t copyStream)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">The host wrapper for copying (packing) the values on the left and right side of the data block to separate, contiguous buffers.  <a href="#a95a52e3c2e58d2d94403a0dce2e29686"></a><br/></td></tr>
</table>
<hr/><a name="details" id="details"></a><h2>Detailed Description</h2>
<div class="textblock"><p>The header containing the most relevant functions for the Jacobi solver. </p>
</div><hr/><h2>Define Documentation</h2>
<a class="anchor" id="ae7bacf72b151d28a9a4c55d10db0fa0b"></a><!-- doxytag: member="Jacobi.h::DEFAULT_DOMAIN_SIZE" ref="ae7bacf72b151d28a9a4c55d10db0fa0b" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#ae7bacf72b151d28a9a4c55d10db0fa0b">DEFAULT_DOMAIN_SIZE</a>&#160;&#160;&#160;4096</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the default domain size (when not explicitly stated with "-d" in the command-line arguments). </p>

</div>
</div>
<a class="anchor" id="a1cf673c1febd6ac97dd8aefadf00b5f7"></a><!-- doxytag: member="Jacobi.h::ENV_LOCAL_RANK" ref="a1cf673c1febd6ac97dd8aefadf00b5f7" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a1cf673c1febd6ac97dd8aefadf00b5f7">ENV_LOCAL_RANK</a>&#160;&#160;&#160;&quot;MV2_COMM_WORLD_LOCAL_RANK&quot;</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the environment variable which allows the reading of the local rank of the current MPI process before the MPI environment gets initialized with MPI_Init(). This is necessary when running the CUDA-aware MPI version of the Jacobi solver, which needs this information in order to be able to set the CUDA device for the MPI process before MPI environment initialization. If you are using MVAPICH2, set this constant to "MV2_COMM_WORLD_LOCAL_RANK"; for Open MPI, use "OMPI_COMM_WORLD_LOCAL_RANK". </p>

</div>
</div>
<a class="anchor" id="a3941d7542ce359c2489df632b4893aee"></a><!-- doxytag: member="Jacobi.h::JACOBI_MAX_LOOPS" ref="a3941d7542ce359c2489df632b4893aee" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a3941d7542ce359c2489df632b4893aee">JACOBI_MAX_LOOPS</a>&#160;&#160;&#160;1000</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the Jacobi iteration count limit. The Jacobi run will never cycle more than this, even if it has not converged when finishing the last allowed iteration. </p>

</div>
</div>
<a class="anchor" id="a3bfda2a5c101b1a0762152713cda5aa8"></a><!-- doxytag: member="Jacobi.h::JACOBI_TOLERANCE" ref="a3bfda2a5c101b1a0762152713cda5aa8" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a3bfda2a5c101b1a0762152713cda5aa8">JACOBI_TOLERANCE</a>&#160;&#160;&#160;1.0E-5F</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the Jacobi tolerance threshold. The run is considered to have converged when the maximum residue falls below this value. </p>

</div>
</div>
<a class="anchor" id="a05512809cad5b2530798ada1a5f06a83"></a><!-- doxytag: member="Jacobi.h::MIN_DOM_SIZE" ref="a05512809cad5b2530798ada1a5f06a83" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a05512809cad5b2530798ada1a5f06a83">MIN_DOM_SIZE</a>&#160;&#160;&#160;3</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the minimum acceptable domain size in any of the 2 dimensions. </p>

</div>
</div>
<a class="anchor" id="a30c87e11fd7fecbcda306bb0b73ef320"></a><!-- doxytag: member="Jacobi.h::MPI_MASTER_RANK" ref="a30c87e11fd7fecbcda306bb0b73ef320" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a30c87e11fd7fecbcda306bb0b73ef320">MPI_MASTER_RANK</a>&#160;&#160;&#160;0</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the global rank of the root (master) process; in this application, it is mostly relevant for printing purposes. </p>

</div>
</div>
<a class="anchor" id="a23205a6a97096d57a247794f5e250963"></a><!-- doxytag: member="Jacobi.h::STATUS_ERR" ref="a23205a6a97096d57a247794f5e250963" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a23205a6a97096d57a247794f5e250963">STATUS_ERR</a>&#160;&#160;&#160;-1</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the status value that indicates an error. </p>

</div>
</div>
<a class="anchor" id="a2403320c41f08e7567cb169de5db66b3"></a><!-- doxytag: member="Jacobi.h::STATUS_OK" ref="a2403320c41f08e7567cb169de5db66b3" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a2403320c41f08e7567cb169de5db66b3">STATUS_OK</a>&#160;&#160;&#160;0</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>This is the status value that indicates a successful operation. </p>

</div>
</div>
<a class="anchor" id="a5a03415e5a2d7bdc6ce680c47ddc156a"></a><!-- doxytag: member="Jacobi.h::USE_FLOAT" ref="a5a03415e5a2d7bdc6ce680c47ddc156a" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">#define <a class="el" href="Jacobi_8h.html#a5a03415e5a2d7bdc6ce680c47ddc156a">USE_FLOAT</a>&#160;&#160;&#160;0</td>
        </tr>
      </table>
</div>
<div class="memdoc">
<p>Setting this to 1 makes the application use only single-precision floating-point data. Set this to 0 in order to use double-precision floating-point data instead. </p>

</div>
</div>
<hr/><h2>Function Documentation</h2>
<a class="anchor" id="aef3118f365ec96030a022fa17b73fc42"></a><!-- doxytag: member="Jacobi.h::ApplyTopology" ref="aef3118f365ec96030a022fa17b73fc42" args="(int *rank, int size, const int2 *topSize, int *neighbors, int2 *topIndex, MPI_Comm *cartComm)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">int <a class="el" href="Jacobi_8h.html#aef3118f365ec96030a022fa17b73fc42">ApplyTopology</a> </td>
          <td>(</td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int2 *&#160;</td>
          <td class="paramname"><em>topIndex</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">MPI_Comm *&#160;</td>
          <td class="paramname"><em>cartComm</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Generates the 2D topology and establishes the neighbor relationships between MPI processes. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in,out]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topSize</td><td>The desired topology size (this must match the number of available MPI processes) </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">neighbors</td><td>The list that will be populated with the direct neighbors of the calling MPI process </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">topIndex</td><td>The 2D index that the calling MPI process will have in the topology </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph.png" border="0" usemap="#Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph" alt=""/></div>
<map name="Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph" id="Jacobi_8h_aef3118f365ec96030a022fa17b73fc42_cgraph">
<area shape="rect" id="node3" href="CUDA__Aware__MPI_8c.html#aa8cb031e8cba7840a96582f530283fd5" title="This allows the MPI process to set the CUDA device after the MPI environment is initialized For the C..." alt="" coords="164,5,292,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="a08324d1c9b1ee7193242670ba4b97d88"></a><!-- doxytag: member="Jacobi.h::CallJacobiKernel" ref="a08324d1c9b1ee7193242670ba4b97d88" args="(real *devBlocks[2], real *devResidue, const int4 *bounds, const int2 *size)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">real <a class="el" href="Jacobi_8h.html#a08324d1c9b1ee7193242670ba4b97d88">CallJacobiKernel</a> </td>
          <td>(</td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devResidue</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int4 *&#160;</td>
          <td class="paramname"><em>bounds</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>size</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>The host wrapper for one Jacobi iteration. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devBlocks</td><td>The 2 blocks involved: the first is the current one, the second is the one to be updated </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devResidue</td><td>The global residue that is to be updated through the Jacobi iteration </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">bounds</td><td>The bounds of the rectangular block region that holds only computable values </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The 2D size of data blocks, excluding the edges which hold the halo values </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph.png" border="0" usemap="#Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph" alt=""/></div>
<map name="Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph" id="Jacobi_8h_a08324d1c9b1ee7193242670ba4b97d88_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="173,5,288,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="a87c5ce86d74764a06a13b73dff013870"></a><!-- doxytag: member="Jacobi.h::CheckCudaCall" ref="a87c5ce86d74764a06a13b73dff013870" args="(cudaError_t command, const char *commandName, const char *fileName, int line)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a87c5ce86d74764a06a13b73dff013870">CheckCudaCall</a> </td>
          <td>(</td>
          <td class="paramtype">cudaError_t&#160;</td>
          <td class="paramname"><em>command</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const char *&#160;</td>
          <td class="paramname"><em>commandName</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const char *&#160;</td>
          <td class="paramname"><em>fileName</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>line</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>The host function for checking the result of a CUDA API call. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">command</td><td>The result of the previously-issued CUDA API call </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">commandName</td><td>The name of the issued API call </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">fileName</td><td>The name of the file where the API call occurred </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">line</td><td>The line in the file where the API call occurred </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a3b641f8b5d521a230416b64d8b809324"></a><!-- doxytag: member="Jacobi.h::CopyDevHalosToBlock" ref="a3b641f8b5d521a230416b64d8b809324" args="(real *devBlock, const real *devHaloLineLeft, const real *devHaloLineRight, const int2 *size, const int *neighbors)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a3b641f8b5d521a230416b64d8b809324">CopyDevHalosToBlock</a> </td>
          <td>(</td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlock</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const real *&#160;</td>
          <td class="paramname"><em>devHaloLineLeft</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const real *&#160;</td>
          <td class="paramname"><em>devHaloLineRight</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>The host wrapper for copying (unpacking) the values from the halo buffers to the left and right side of the data block. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[out]</td><td class="paramname">devBlock</td><td>The 2D device block that will contain the halo values after unpacking </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devHaloLineLeft</td><td>The halo buffer for the left side of the data block </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devHaloLineRight</td><td>The halo buffer for the right side of the data block </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The 2D size of data block, excluding the edges which hold the halo values </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The ranks of the neighboring MPI processes </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph.png" border="0" usemap="#Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph" alt=""/></div>
<map name="Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph" id="Jacobi_8h_a3b641f8b5d521a230416b64d8b809324_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="213,5,328,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="aeaebe085afbffaa0d6bcd77fa16ceb97"></a><!-- doxytag: member="Jacobi.h::CopyDeviceBlock" ref="aeaebe085afbffaa0d6bcd77fa16ceb97" args="(real *devBlocks[2], const int4 *bounds, const int2 *size)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#aeaebe085afbffaa0d6bcd77fa16ceb97">CopyDeviceBlock</a> </td>
          <td>(</td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int4 *&#160;</td>
          <td class="paramname"><em>bounds</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>size</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>The host wrapper for copying the updated block over the old one, after a Jacobi iteration finishes. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devBlocks</td><td>The 2 blocks involved: the first is the old one, the second is the updated one </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">bounds</td><td>The bounds of the rectangular updated region (holding only computable values) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The 2D size of data blocks, excluding the edges which hold the halo values </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph.png" border="0" usemap="#Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph" alt=""/></div>
<map name="Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph" id="Jacobi_8h_aeaebe085afbffaa0d6bcd77fa16ceb97_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="181,5,296,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="a95a52e3c2e58d2d94403a0dce2e29686"></a><!-- doxytag: member="Jacobi.h::CopyDevSideEdgesFromBlock" ref="a95a52e3c2e58d2d94403a0dce2e29686" args="(const real *devBlock, real *devSideEdges[2], const int2 *size, const int *neighbors, cudaStream_t copyStream)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a95a52e3c2e58d2d94403a0dce2e29686">CopyDevSideEdgesFromBlock</a> </td>
          <td>(</td>
          <td class="paramtype">const real *&#160;</td>
          <td class="paramname"><em>devBlock</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t&#160;</td>
          <td class="paramname"><em>copyStream</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>The host wrapper for copying (packing) the values on the left and right side of the data block to separate, contiguous buffers. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">devBlock</td><td>The 2D device block containing the updated values after a Jacobi run </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devSideEdges</td><td>The buffers where the edge values will be packed in </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The 2D size of data block, excluding the edges which hold the halo values for the next iteration </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The ranks of the neighboring MPI processes </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream on which this kernel will be executed </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph.png" border="0" usemap="#Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph" alt=""/></div>
<map name="Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph" id="Jacobi_8h_a95a52e3c2e58d2d94403a0dce2e29686_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="256,5,371,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="a1dc0674ebcbca5a781885f2204d78251"></a><!-- doxytag: member="Jacobi.h::ExchangeHalos" ref="a1dc0674ebcbca5a781885f2204d78251" args="(MPI_Comm cartComm, real *devSend, real *hostSend, real *hostRecv, real *devRecv, int neighbor, int elemCount)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a1dc0674ebcbca5a781885f2204d78251">ExchangeHalos</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSend</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSend</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecv</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devRecv</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>neighbor</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>elemCount</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Exchange halo values between 2 direct neighbors This is the main difference between the normal CUDA &amp; MPI version and the CUDA-aware MPI version. In the former, the exchange first requires a copy from device to host memory, an MPI call using the host buffer and lastly, a copy of the received host buffer back to the device memory. In the latter, the host buffers are completely skipped, as the MPI environment uses the device buffers directly. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devSend</td><td>The device buffer that needs to be sent </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostSend</td><td>The host buffer where the device buffer is first copied to (not needed here) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostRecv</td><td>The host buffer that receives the halo values (not needed here) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devRecv</td><td>The device buffer that receives the halo buffers directly </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbor</td><td>The rank of the neighbor MPI process in the carthesian communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">elemCount</td><td>The number of elements to transfer</td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devSend</td><td>The device buffer that needs to be sent </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostSend</td><td>The host buffer where the device buffer is first copied to </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostRecv</td><td>The host buffer that receives the halo values directly </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devRecv</td><td>The device buffer where the receiving host buffer is copied to </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbor</td><td>The rank of the neighbor MPI process in the carthesian communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">elemCount</td><td>The number of elements to transfer </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="ad25cf65f450abbc68de1a6409b0c92ba"></a><!-- doxytag: member="Jacobi.h::Finalize" ref="ad25cf65f450abbc68de1a6409b0c92ba" args="(real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#ad25cf65f450abbc68de1a6409b0c92ba">Finalize</a> </td>
          <td>(</td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devResidue</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t&#160;</td>
          <td class="paramname"><em>copyStream</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Close (finalize) the MPI environment and deallocate buffers. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that were used during the Jacobi iterations </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devSideEdges</td><td>The 2 device side edges that were used to hold updated halos before sending </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devHaloLines</td><td>The 2 device lines that were used to hold received halos </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that were used at halo exchange in the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that were used at halo exchange in the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devResidue</td><td>The global residue, kept in device memory </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a0ea9f99f85b380e912940f351366a862"></a><!-- doxytag: member="Jacobi.h::Initialize" ref="a0ea9f99f85b380e912940f351366a862" args="(int *argc, char ***argv, int *rank, int *size)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a0ea9f99f85b380e912940f351366a862">Initialize</a> </td>
          <td>(</td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>argc</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">char ***&#160;</td>
          <td class="paramname"><em>argv</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>size</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Initialize the MPI environment, allowing the CUDA device to be selected before (if necessary) </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in,out]</td><td class="paramname">argc</td><td>The number of command-line arguments </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">argv</td><td>The list of command-line arguments </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">rank</td><td>The global rank of the current MPI process </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph.png" border="0" usemap="#Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph" alt=""/></div>
<map name="Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph" id="Jacobi_8h_a0ea9f99f85b380e912940f351366a862_cgraph">
<area shape="rect" id="node3" href="CUDA__Aware__MPI_8c.html#ac7925a5469b04c01679b20577671c975" title="This allows the MPI process to set the CUDA device before the MPI environment is initialized For the ..." alt="" coords="125,5,264,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="a63442a950638ea4accb5dcb5ad7a8b87"></a><!-- doxytag: member="Jacobi.h::InitializeDataChunk" ref="a63442a950638ea4accb5dcb5ad7a8b87" args="(int topSizeY, int topIdxY, const int2 *domSize, const int *neighbors, cudaStream_t *copyStream, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real **devResidue)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a63442a950638ea4accb5dcb5ad7a8b87">InitializeDataChunk</a> </td>
          <td>(</td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>topSizeY</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>topIdxY</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t *&#160;</td>
          <td class="paramname"><em>copyStream</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real **&#160;</td>
          <td class="paramname"><em>devResidue</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This allocates and initializes all the relevant data buffers before the Jacobi run. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">topSizeY</td><td>The size of the topology in the Y direction </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topIdxY</td><td>The Y index of the calling MPI process in the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The size of the local domain (for which only the current MPI process is responsible) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The neighbor ranks, according to the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that will be updated during the Jacobi run </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devSideEdges</td><td>The 2 side edges (parallel to the Y direction) that will hold the packed halo values before sending them </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devHaloLines</td><td>The 2 halo lines (parallel to the Y direction) that will hold the packed halo values after receiving them </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that will be used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that will be used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devResidue</td><td>The global device residue, which will be updated after every Jacobi iteration </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a243a9fb9128b70313b30d3f082cc0a61"></a><!-- doxytag: member="Jacobi.h::ParseCommandLineArguments" ref="a243a9fb9128b70313b30d3f082cc0a61" args="(int argc, char **argv, int rank, int size, int2 *domSize, int2 *topSize, int *useFastSwap)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">int <a class="el" href="Jacobi_8h.html#a243a9fb9128b70313b30d3f082cc0a61">ParseCommandLineArguments</a> </td>
          <td>(</td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>argc</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">char **&#160;</td>
          <td class="paramname"><em>argv</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int2 *&#160;</td>
          <td class="paramname"><em>topSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>useFastSwap</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Parses the application's command-line arguments. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">argc</td><td>The number of input arguments </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">argv</td><td>The input arguments </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The MPI rank of the calling process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">domSize</td><td>The parsed domain size (2D) </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">topSize</td><td>The parsed topology size (2D) </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">useFastSwap</td><td>The parsed flag for fast block swap </td></tr>
  </table>
  </dd>
</dl>
<dl class="return"><dt><b>Returns:</b></dt><dd>The parsing status (STATUS_OK indicates a successful parse) </dd></dl>

</div>
</div>
<a class="anchor" id="a29316e4a0e49080875975ead1809fdbb"></a><!-- doxytag: member="Jacobi.h::PostRunJacobi" ref="a29316e4a0e49080875975ead1809fdbb" args="(MPI_Comm cartComm, int rank, int size, const int2 *topSize, const int2 *domSize, int iterations, int useFastSwap, double timerStart, double avgTransferTime)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a29316e4a0e49080875975ead1809fdbb">PostRunJacobi</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>iterations</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>useFastSwap</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double&#160;</td>
          <td class="paramname"><em>timerStart</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double&#160;</td>
          <td class="paramname"><em>avgTransferTime</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This function is called immediately after the main Jacobi loop. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topSize</td><td>The size of the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The size of the local domain </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">iterations</td><td>The number of successfully completed Jacobi iterations </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">useFastSwap</td><td>The flag indicating if fast pointer swapping was used to exchange blocks </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">timerStart</td><td>The Jacobi loop starting moment (measured as wall-time) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">avgTransferTime</td><td>The average time spent performing MPI transfers (per process) </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a053b144f9d3b11ad9b3563447876629f"></a><!-- doxytag: member="Jacobi.h::PreRunJacobi" ref="a053b144f9d3b11ad9b3563447876629f" args="(MPI_Comm cartComm, int rank, int size, double *timerStart)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a053b144f9d3b11ad9b3563447876629f">PreRunJacobi</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double *&#160;</td>
          <td class="paramname"><em>timerStart</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This function is called immediately before the main Jacobi loop. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">timerStart</td><td>The Jacobi loop starting moment (measured as wall-time) </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a52ea6f92f186de790a55fdf700026dd9"></a><!-- doxytag: member="Jacobi.h::RunJacobi" ref="a52ea6f92f186de790a55fdf700026dd9" args="(MPI_Comm cartComm, int rank, int size, const int2 *domSize, const int2 *topIndex, const int *neighbors, int useFastSwap, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream, int *iterations, double *avgTransferTime)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a52ea6f92f186de790a55fdf700026dd9">RunJacobi</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topIndex</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>useFastSwap</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devResidue</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t&#160;</td>
          <td class="paramname"><em>copyStream</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>iterations</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double *&#160;</td>
          <td class="paramname"><em>avgTransferTime</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This is the main Jacobi loop, which handles device computation and data exchange between MPI processes. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The number of available MPI processes </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The 2D size of the local domain </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topIndex</td><td>The 2D index of the calling MPI process in the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The list of ranks which are direct neighbors to the caller </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">useFastSwap</td><td>This flag indicates if blocks should be swapped through pointer copy (faster) or through element-by-element copy (slower) </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that are updated during the Jacobi run </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devSideEdges</td><td>The 2 side edges (parallel to the Y direction) that hold the packed halo values before sending them </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devHaloLines</td><td>The 2 halo lines (parallel to the Y direction) that hold the packed halo values after receiving them </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that are used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that are used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devResidue</td><td>The global device residue, which gets updated after every Jacobi iteration </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">iterations</td><td>The number of successfully completed iterations </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">avgTransferTime</td><td>The average time spent performing MPI transfers (per process) </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph.png" border="0" usemap="#Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph" alt=""/></div>
<map name="Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph" id="Jacobi_8h_a52ea6f92f186de790a55fdf700026dd9_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a08324d1c9b1ee7193242670ba4b97d88" title="The host wrapper for one Jacobi iteration." alt="" coords="143,5,263,35"/><area shape="rect" id="node7" href="Device_8cu.html#aeaebe085afbffaa0d6bcd77fa16ceb97" title="The host wrapper for copying the updated block over the old one, after a Jacobi iteration finishes..." alt="" coords="140,59,265,88"/><area shape="rect" id="node10" href="Host_8c.html#ac28696e075d2053b4de5662556c16f3d" title="This performs the exchanging of all necessary halos between 2 neighboring MPI processes." alt="" coords="144,137,261,167"/><area shape="rect" id="node5" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="565,83,680,112"/><area shape="rect" id="node12" href="Device_8cu.html#a95a52e3c2e58d2d94403a0dce2e29686" title="The host wrapper for copying (packing) the values on the left and right side of the data block to sep..." alt="" coords="315,108,517,137"/><area shape="rect" id="node15" href="CUDA__Aware__MPI_8c.html#a1dc0674ebcbca5a781885f2204d78251" title="Exchange halo values between 2 direct neighbors This is the main difference between the normal CUDA &amp;..." alt="" coords="359,161,473,191"/><area shape="rect" id="node17" href="Device_8cu.html#a3b641f8b5d521a230416b64d8b809324" title="The host wrapper for copying (unpacking) the values from the halo buffers to the left and right side ..." alt="" coords="337,215,495,244"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="aa8cb031e8cba7840a96582f530283fd5"></a><!-- doxytag: member="Jacobi.h::SetDeviceAfterInit" ref="aa8cb031e8cba7840a96582f530283fd5" args="(int rank)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#aa8cb031e8cba7840a96582f530283fd5">SetDeviceAfterInit</a> </td>
          <td>(</td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em></td><td>)</td>
          <td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This allows the MPI process to set the CUDA device after the MPI environment is initialized For the CUDA-aware MPI version, there is nothing to be done here. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The global rank of the calling MPI process</td></tr>
  </table>
  </dd>
</dl>
<p>This allows the MPI process to set the CUDA device after the MPI environment is initialized For the CUDA-aware MPI version, there is nothing to be done here.</p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The global rank of the calling MPI process </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
</div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Fri Mar 1 2013 15:52:54 for Jacobi by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
